{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import bert\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "from bert import modeling\n",
    "import numpy as np\n",
    "import json\n",
    "import tensorflow as tf\n",
    "import itertools\n",
    "import collections\n",
    "import re\n",
    "import random\n",
    "import sentencepiece as spm\n",
    "from unidecode import unidecode\n",
    "from sklearn.utils import shuffle\n",
    "from tqdm import tqdm\n",
    "from prepro_utils import preprocess_text, encode_ids, encode_pieces\n",
    "from malaya.text.function import transformer_textcleaning as cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('sp10m.cased.bert.model')\n",
    "\n",
    "with open('sp10m.cased.bert.vocab') as fopen:\n",
    "    v = fopen.read().split('\\n')[:-1]\n",
    "v = [i.split('\\t') for i in v]\n",
    "v = {i[0]: i[1] for i in v}\n",
    "\n",
    "\n",
    "class Tokenizer:\n",
    "    def __init__(self, v, sp_model):\n",
    "        self.vocab = v\n",
    "        self.sp_model = sp_model\n",
    "\n",
    "    def tokenize(self, string):\n",
    "        return encode_pieces(\n",
    "            self.sp_model, string, return_unicode = False, sample = False\n",
    "        )\n",
    "\n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return [self.sp_model.PieceToId(piece) for piece in tokens]\n",
    "\n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return [self.sp_model.IdToPiece(i) for i in ids]\n",
    "\n",
    "\n",
    "tokenizer = Tokenizer(v, sp_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('train_X.pkl', 'rb') as fopen:\n",
    "    train_X, train_Y, train_depends = pickle.load(fopen)\n",
    "    \n",
    "with open('test_X.pkl', 'rb') as fopen:\n",
    "    test_X, test_Y, test_depends = pickle.load(fopen)\n",
    "    \n",
    "with open('tags.pkl', 'rb') as fopen:\n",
    "    idx2tag, tag2idx = pickle.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/modeling.py:93: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bert_config = modeling.BertConfig.from_json_file(\n",
    "    'tiny-bert-v1/config.json'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_INIT_CHKPNT = 'tiny-bert-v1/model.ckpt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 3\n",
    "batch_size = 32\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(len(train_X) / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiAAttention:\n",
    "    def __init__(self, input_size_encoder, input_size_decoder, num_labels):\n",
    "        self.input_size_encoder = input_size_encoder\n",
    "        self.input_size_decoder = input_size_decoder\n",
    "        self.num_labels = num_labels\n",
    "        \n",
    "        self.W_d = tf.get_variable(\"W_d\", shape=[self.num_labels, self.input_size_decoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_e = tf.get_variable(\"W_e\", shape=[self.num_labels, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.U = tf.get_variable(\"U\", shape=[self.num_labels, self.input_size_decoder, self.input_size_encoder],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        \n",
    "    def forward(self, input_d, input_e, mask_d=None, mask_e=None):\n",
    "        batch = tf.shape(input_d)[0]\n",
    "        length_decoder = tf.shape(input_d)[1]\n",
    "        length_encoder = tf.shape(input_e)[1]\n",
    "        out_d = tf.expand_dims(tf.matmul(self.W_d, tf.transpose(input_d, [0, 2, 1])), 3)\n",
    "        out_e = tf.expand_dims(tf.matmul(self.W_e, tf.transpose(input_e, [0, 2, 1])), 2)\n",
    "        output = tf.matmul(tf.expand_dims(input_d, 1), self.U)\n",
    "        output = tf.matmul(output, tf.transpose(tf.expand_dims(input_e, 1), [0, 1, 3, 2]))\n",
    "        \n",
    "        output = output + out_d + out_e\n",
    "        \n",
    "        if mask_d is not None:\n",
    "            d = tf.expand_dims(tf.expand_dims(mask_d, 1), 3)\n",
    "            e = tf.expand_dims(tf.expand_dims(mask_e, 1), 2)\n",
    "            output = output * d * e\n",
    "            \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiLinear:\n",
    "    def __init__(self, left_features, right_features, out_features):\n",
    "        self.left_features = left_features\n",
    "        self.right_features = right_features\n",
    "        self.out_features = out_features\n",
    "        \n",
    "        self.U = tf.get_variable(\"U-bi\", shape=[out_features, left_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_l = tf.get_variable(\"Wl\", shape=[out_features, left_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "        self.W_r = tf.get_variable(\"Wr\", shape=[out_features, right_features],\n",
    "           initializer=tf.contrib.layers.xavier_initializer())\n",
    "    \n",
    "    def forward(self, input_left, input_right):\n",
    "        left_size = tf.shape(input_left)\n",
    "        output_shape = tf.concat([left_size[:-1], [self.out_features]], axis = 0)\n",
    "        batch = tf.cast(tf.reduce_prod(left_size[:-1]), tf.int32)\n",
    "        input_left = tf.reshape(input_left, (batch, self.left_features))\n",
    "        input_right = tf.reshape(input_right, (batch, self.right_features))\n",
    "        tiled = tf.tile(tf.expand_dims(input_left, axis = 0), (self.out_features,1,1))\n",
    "        output = tf.transpose(tf.reduce_sum(tf.matmul(tiled, self.U), axis = 2))\n",
    "        output = output + tf.matmul(input_left, tf.transpose(self.W_l))\\\n",
    "        + tf.matmul(input_right, tf.transpose(self.W_r))\n",
    "        \n",
    "        return tf.reshape(output, output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "_NEG_INF = -1e9\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        learning_rate,\n",
    "        hidden_size_word,\n",
    "        training = True,\n",
    "        cov = 0.0):\n",
    "        \n",
    "        self.words = tf.placeholder(tf.int32, (None, None))\n",
    "        self.heads = tf.placeholder(tf.int32, (None, None))\n",
    "        self.types = tf.placeholder(tf.int32, (None, None))\n",
    "        self.switch = tf.placeholder(tf.bool, None)\n",
    "        self.mask = tf.cast(tf.math.not_equal(self.words, 0), tf.float32)\n",
    "        self.maxlen = tf.shape(self.words)[1]\n",
    "        self.lengths = tf.count_nonzero(self.words, 1)\n",
    "        mask = self.mask\n",
    "        heads = self.heads\n",
    "        types = self.types\n",
    "        \n",
    "        self.arc_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.arc_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.attention = BiAAttention(hidden_size_word, hidden_size_word, 1)\n",
    "\n",
    "        self.type_h = tf.layers.Dense(hidden_size_word)\n",
    "        self.type_c = tf.layers.Dense(hidden_size_word)\n",
    "        self.bilinear = BiLinear(hidden_size_word, hidden_size_word, len(tag2idx))\n",
    "        \n",
    "        model = modeling.BertModel(\n",
    "            config=bert_config,\n",
    "            is_training=training,\n",
    "            input_ids=self.words,\n",
    "            input_mask=self.mask,\n",
    "            use_one_hot_embeddings=False)\n",
    "        \n",
    "        output_layer = model.get_sequence_output()\n",
    "        \n",
    "        arc_h = tf.nn.elu(self.arc_h(output_layer))\n",
    "        arc_c = tf.nn.elu(self.arc_c(output_layer))\n",
    "        self._arc_h = arc_h\n",
    "        self._arc_c = arc_c\n",
    "        \n",
    "        type_h = tf.nn.elu(self.type_h(output_layer))\n",
    "        type_c = tf.nn.elu(self.type_c(output_layer))\n",
    "        self._type_h = type_h\n",
    "        self._type_c = type_c\n",
    "        \n",
    "        out_arc = tf.squeeze(self.attention.forward(arc_h, arc_c, mask_d=self.mask, \n",
    "                                                    mask_e=self.mask), axis = 1)\n",
    "        self.out_arc = out_arc\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        sec_max_len = tf.shape(out_arc)[2]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        \n",
    "        decode_arc = out_arc + tf.linalg.diag(tf.fill([max_len], -np.inf))\n",
    "        minus_mask = tf.expand_dims(tf.cast(1 - mask, tf.bool), axis = 2)\n",
    "        minus_mask = tf.tile(minus_mask, [1, 1, sec_max_len])\n",
    "        decode_arc = tf.where(minus_mask, tf.fill(tf.shape(decode_arc), -np.inf), decode_arc)\n",
    "        self.decode_arc = decode_arc\n",
    "        self.heads_seq = tf.argmax(decode_arc, axis = 1)\n",
    "        self.heads_seq = tf.identity(self.heads_seq, name = 'heads_seq')\n",
    "        \n",
    "#         self.decode_arc_t = tf.transpose(decode_arc, (0, 2, 1))\n",
    "#         sequence_loss_depends = tf.contrib.seq2seq.sequence_loss(logits = self.decode_arc_t,\n",
    "#                                                      targets = self.heads,\n",
    "#                                                      weights = mask)\n",
    "        \n",
    "        t = tf.cast(tf.transpose(self.heads_seq), tf.int32)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        self.tags_seq = tf.argmax(out_type, axis = 2)\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'tags_seq')\n",
    "        \n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            out_type, self.types, self.lengths\n",
    "        )\n",
    "        crf_loss = tf.reduce_mean(-log_likelihood)\n",
    "        self.logits, _ = tf.contrib.crf.crf_decode(\n",
    "            out_type, transition_params, self.lengths\n",
    "        )\n",
    "        self.logits = tf.identity(self.logits, name = 'logits')\n",
    "        \n",
    "        batch = tf.shape(out_arc)[0]\n",
    "        max_len = tf.shape(out_arc)[1]\n",
    "        batch_index = tf.range(0, batch)\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0), \n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        type_h = tf.gather_nd(type_h, concatenated)\n",
    "        out_type = self.bilinear.forward(type_h, type_c)\n",
    "        minus_inf = -1e8\n",
    "        minus_mask = (1 - mask) * minus_inf\n",
    "        out_arc = out_arc + tf.expand_dims(minus_mask, axis = 2) + tf.expand_dims(minus_mask, axis = 1)\n",
    "        loss_arc = tf.nn.log_softmax(out_arc, dim=1)\n",
    "        loss_type = tf.nn.log_softmax(out_type, dim=2)\n",
    "        loss_arc = loss_arc * tf.expand_dims(mask, axis = 2) * tf.expand_dims(mask, axis = 1)\n",
    "        loss_type = loss_type * tf.expand_dims(mask, axis = 2)\n",
    "        num = tf.reduce_sum(mask) - tf.cast(batch, tf.float32)\n",
    "        child_index = tf.tile(tf.expand_dims(tf.range(0, max_len), 1), [1, batch])\n",
    "        t = tf.transpose(heads)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0)], axis = 0))\n",
    "        loss_arc = tf.gather_nd(loss_arc, concatenated)\n",
    "        loss_arc = tf.transpose(loss_arc, [1, 0])\n",
    "        \n",
    "        t = tf.transpose(types)\n",
    "        broadcasted = tf.broadcast_to(batch_index, tf.shape(t))\n",
    "        concatenated = tf.transpose(tf.concat([tf.expand_dims(broadcasted, axis = 0),\n",
    "                                               tf.expand_dims(child_index, axis = 0),\n",
    "                                               tf.expand_dims(t, axis = 0)], axis = 0))\n",
    "        loss_type = tf.gather_nd(loss_type, concatenated)\n",
    "        loss_type = tf.transpose(loss_type, [1, 0])\n",
    "        cost = (tf.reduce_sum(-loss_arc) / num) + (tf.reduce_sum(-loss_type) / num)\n",
    "        \n",
    "        self.cost = tf.cond(self.switch, lambda: cost + crf_loss, lambda: cost)\n",
    "        self.optimizer = optimization.create_optimizer(self.cost, learning_rate, \n",
    "                                                       num_train_steps, num_warmup_steps, False)\n",
    "        \n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        \n",
    "        self.prediction = tf.boolean_mask(self.logits, mask)\n",
    "        mask_label = tf.boolean_mask(self.types, mask)\n",
    "        correct_pred = tf.equal(tf.cast(self.prediction, tf.int32), mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "        self.prediction = tf.cast(tf.boolean_mask(self.heads_seq, mask), tf.int32)\n",
    "        mask_label = tf.boolean_mask(self.heads, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy_depends = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/modeling.py:171: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/modeling.py:490: The name tf.assert_less_equal is deprecated. Please use tf.compat.v1.assert_less_equal instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/modeling.py:358: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/modeling.py:671: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:From <ipython-input-10-66c1ff637216>:61: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/crf/python/ops/crf.py:213: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From <ipython-input-10-66c1ff637216>:101: calling log_softmax (from tensorflow.python.ops.nn_ops) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/optimization.py:27: The name tf.train.get_or_create_global_step is deprecated. Please use tf.compat.v1.train.get_or_create_global_step instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/optimization.py:32: The name tf.train.polynomial_decay is deprecated. Please use tf.compat.v1.train.polynomial_decay instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/bert-standard/bert/optimization.py:70: The name tf.trainable_variables is deprecated. Please use tf.compat.v1.trainable_variables instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "model = Model(learning_rate, hidden_size_word)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from tiny-bert-v1/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, BERT_INIT_CHKPNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "batch_x = train_X[:5]\n",
    "batch_x = pad_sequences(batch_x,padding='post')\n",
    "batch_y = train_Y[:5]\n",
    "batch_y = pad_sequences(batch_y,padding='post')\n",
    "batch_depends = train_depends[:5]\n",
    "batch_depends = pad_sequences(batch_depends,padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.01369863, 0.09589041, 30.746227]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.switch: False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.01369863, 0.09589041, 157.3611]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run([model.accuracy, model.accuracy_depends, model.cost],\n",
    "        feed_dict = {model.words: batch_x,\n",
    "                model.types: batch_y,\n",
    "                model.heads: batch_depends,\n",
    "                model.switch: True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:   8%|▊         | 8228/97788 [21:47<3:47:18,  6.57it/s, accuracy=0.815, accuracy_depends=0.494, cost=1.85] IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "train minibatch loop: 100%|██████████| 97788/97788 [4:20:01<00:00,  6.27it/s, accuracy=0.884, accuracy_depends=0.755, cost=0.702]   \n",
      "test minibatch loop: 100%|██████████| 313/313 [00:36<00:00,  8.65it/s, accuracy=0.89, accuracy_depends=0.741, cost=0.799] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 1.250804, training acc: 0.847605, training depends: 0.668640, valid loss: 0.735416, valid acc: 0.884954, valid depends: 0.744215\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 97788/97788 [4:17:28<00:00,  6.33it/s, accuracy=0.886, accuracy_depends=0.771, cost=0.613]  \n",
      "test minibatch loop: 100%|██████████| 313/313 [00:35<00:00,  8.70it/s, accuracy=0.89, accuracy_depends=0.743, cost=0.689] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, training loss: 0.664987, training acc: 0.888118, training depends: 0.757840, valid loss: 0.641807, valid acc: 0.889845, valid depends: 0.765465\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epoch = 2\n",
    "\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: False\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'tiny-bert-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 97788/97788 [4:22:12<00:00,  6.22it/s, accuracy=0.886, accuracy_depends=0.788, cost=0.576]   \n",
      "test minibatch loop: 100%|██████████| 313/313 [00:36<00:00,  8.61it/s, accuracy=0.9, accuracy_depends=0.778, cost=0.629]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, training loss: 0.602583, training acc: 0.890296, training depends: 0.772322, valid loss: 0.611718, valid acc: 0.891331, valid depends: 0.773028\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 97788/97788 [4:07:31<00:00,  6.58it/s, accuracy=0.891, accuracy_depends=0.783, cost=0.613]  \n",
      "test minibatch loop: 100%|██████████| 313/313 [00:34<00:00,  9.09it/s, accuracy=0.885, accuracy_depends=0.788, cost=0.621]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, training loss: 0.587275, training acc: 0.891074, training depends: 0.775791, valid loss: 0.609368, valid acc: 0.890627, valid depends: 0.773412\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "epoch = 1\n",
    "\n",
    "for e in range(epoch):\n",
    "    train_acc, train_loss = [], []\n",
    "    test_acc, test_loss = [], []\n",
    "    train_acc_depends, test_acc_depends = [], []\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = train_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = train_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost, _ = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost, model.optimizer],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            train_loss.append(cost)\n",
    "            train_acc.append(acc)\n",
    "            train_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "        \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i: index]\n",
    "        batch_x = pad_sequences(batch_x,padding='post')\n",
    "        batch_y = test_Y[i: index]\n",
    "        batch_y = pad_sequences(batch_y,padding='post')\n",
    "        batch_depends = test_depends[i: index]\n",
    "        batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "        \n",
    "        if batch_x.shape == batch_y.shape:\n",
    "            acc_depends, acc, cost = sess.run(\n",
    "                [model.accuracy_depends, model.accuracy, model.cost],\n",
    "                feed_dict = {\n",
    "                    model.words: batch_x,\n",
    "                    model.types: batch_y,\n",
    "                    model.heads: batch_depends,\n",
    "                    model.switch: True\n",
    "                },\n",
    "            )\n",
    "            test_loss.append(cost)\n",
    "            test_acc.append(acc)\n",
    "            test_acc_depends.append(acc_depends)\n",
    "            pbar.set_postfix(cost = cost, accuracy = acc, accuracy_depends = acc_depends)\n",
    "    \n",
    "    \n",
    "    print(\n",
    "    'epoch: %d, training loss: %f, training acc: %f, training depends: %f, valid loss: %f, valid acc: %f, valid depends: %f\\n'\n",
    "    % (e, np.mean(train_loss), \n",
    "       np.mean(train_acc), \n",
    "       np.mean(train_acc_depends), \n",
    "       np.mean(test_loss), \n",
    "       np.mean(test_acc), \n",
    "       np.mean(test_acc_depends)\n",
    "    ))\n",
    "    \n",
    "    saver = tf.train.Saver(tf.trainable_variables())\n",
    "    saver.save(sess, 'tiny-bert-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_sentencepiece_tokens_tagging(x, y):\n",
    "    new_paired_tokens = []\n",
    "    n_tokens = len(x)\n",
    "    rejected = ['[CLS]', '[SEP]']\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    while i < n_tokens:\n",
    "\n",
    "        current_token, current_label = x[i], y[i]\n",
    "        if not current_token.startswith('▁') and current_token not in rejected:\n",
    "            previous_token, previous_label = new_paired_tokens.pop()\n",
    "            merged_token = previous_token\n",
    "            merged_label = [previous_label]\n",
    "            while (\n",
    "                not current_token.startswith('▁')\n",
    "                and current_token not in rejected\n",
    "            ):\n",
    "                merged_token = merged_token + current_token.replace('▁', '')\n",
    "                merged_label.append(current_label)\n",
    "                i = i + 1\n",
    "                current_token, current_label = x[i], y[i]\n",
    "            merged_label = merged_label[0]\n",
    "            new_paired_tokens.append((merged_token, merged_label))\n",
    "\n",
    "        else:\n",
    "            new_paired_tokens.append((current_token, current_label))\n",
    "            i = i + 1\n",
    "\n",
    "    words = [\n",
    "        i[0].replace('▁', '')\n",
    "        for i in new_paired_tokens\n",
    "        if i[0] not in rejected\n",
    "    ]\n",
    "    labels = [i[1] for i in new_paired_tokens if i[0] not in rejected]\n",
    "    return words, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "from malaya.function.parse_dependency import DependencyGraph\n",
    "\n",
    "PUNCTUATION = '!\"#$%&\\'()*+,./:;<=>?@[\\]^_`{|}~'\n",
    "\n",
    "def transformer_textcleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    string = re.sub('\\\\(dot\\\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    string = ' '.join(string)\n",
    "    string = re.sub(f'([{PUNCTUATION}])', r' \\1 ', string)\n",
    "    string = re.sub('\\s{2,}', ' ', string)\n",
    "    original_string = string.split()\n",
    "    string = [\n",
    "        (original_string[no], word.title() if word.isupper() else word)\n",
    "        for no, word in enumerate(string.split())\n",
    "        if len(word)\n",
    "    ]\n",
    "    return [s[0] for s in string], [s[1] for s in string]\n",
    "\n",
    "def parse_X(left):\n",
    "    bert_tokens = ['[CLS]']\n",
    "    for no, orig_token in enumerate(left):\n",
    "        t = tokenizer.tokenize(orig_token)\n",
    "        bert_tokens.extend(t)\n",
    "    bert_tokens.append(\"[SEP]\")\n",
    "    t = tokenizer.convert_tokens_to_ids(bert_tokens)\n",
    "    return t, bert_tokens, [1] * len(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dependency_graph(tagging, indexing):\n",
    "    \"\"\"\n",
    "    Return helper object for dependency parser results. Only accept tagging and indexing outputs from dependency models.\n",
    "    \"\"\"\n",
    "    result = []\n",
    "    for i in range(len(tagging)):\n",
    "        result.append(\n",
    "            '%d\\t%s\\t_\\t_\\t_\\t_\\t%d\\t%s\\t_\\t_'\n",
    "            % (i + 1, tagging[i][0], int(indexing[i][1]), tagging[i][1])\n",
    "        )\n",
    "    return DependencyGraph('\\n'.join(result), top_relation_label='root')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"193pt\" height=\"218pt\"\n",
       " viewBox=\"0.00 0.00 192.50 218.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 214)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-214 188.5,-214 188.5,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"94\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"94\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (makan)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M94,-173.9735C94,-162.1918 94,-146.5607 94,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"97.5001,-133.0033 94,-123.0034 90.5001,-133.0034 97.5001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"117.5\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">xcomp</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (husein)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;1 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>2&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M83.64,-86.9735C76.6677,-74.8418 67.3498,-58.6287 59.4999,-44.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"62.3642,-42.9295 54.3468,-36.0034 56.2951,-46.4175 62.3642,-42.9295\"/>\n",
       "<text text-anchor=\"middle\" x=\"92\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"145\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (ayam)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M105.3041,-86.7532C108.7732,-81.0866 112.5772,-74.8047 116,-69 120.611,-61.1804 125.5263,-52.6241 129.9752,-44.7918\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"133.0613,-46.4448 134.9374,-36.0174 126.9682,-42.9989 133.0613,-46.4448\"/>\n",
       "<text text-anchor=\"middle\" x=\"135\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7f34bab49b70>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'husein makan ayam'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, bert_sequence, mask = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(bert_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(bert_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: G Pages: 1 -->\n",
       "<svg width=\"3019pt\" height=\"740pt\"\n",
       " viewBox=\"0.00 0.00 3018.50 740.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 736)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-736 3014.5,-736 3014.5,4 -4,4\"/>\n",
       "<!-- 0 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>0</title>\n",
       "<text text-anchor=\"middle\" x=\"783\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0 (None)</text>\n",
       "</g>\n",
       "<!-- 13 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>13</title>\n",
       "<text text-anchor=\"middle\" x=\"783\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">13 (membidas)</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;13 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>0&#45;&gt;13</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M783,-695.9735C783,-684.1918 783,-668.5607 783,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"786.5001,-655.0033 783,-645.0034 779.5001,-655.0034 786.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"802\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1</title>\n",
       "<text text-anchor=\"middle\" x=\"243\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1 (Kuala)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;1 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>13&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M721.3182,-617.0624C613.6066,-599.7088 393.2345,-564.2044 293.7403,-548.1748\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"294.0712,-544.6831 283.6418,-546.5478 292.9577,-551.594 294.0712,-544.6831\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 14 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>14</title>\n",
       "<text text-anchor=\"middle\" x=\"491\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">14 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;14 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>13&#45;&gt;14</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M722.4974,-608.9735C675.5151,-594.9754 610.2951,-575.5434 561.0508,-560.8713\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"562.0082,-557.5046 551.4251,-558.0034 560.0093,-564.2131 562.0082,-557.5046\"/>\n",
       "<text text-anchor=\"middle\" x=\"666\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 20 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>20</title>\n",
       "<text text-anchor=\"middle\" x=\"701\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">20 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;20 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>13&#45;&gt;20</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M765.0995,-608.7772C759.5838,-603.1116 753.513,-596.8241 748,-591 740.222,-582.783 731.8214,-573.7268 724.3033,-565.5545\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"726.7543,-563.0488 717.4147,-558.0464 721.5963,-567.7812 726.7543,-563.0488\"/>\n",
       "<text text-anchor=\"middle\" x=\"763\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 33 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>33</title>\n",
       "<text text-anchor=\"middle\" x=\"879\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">33 (melaksanakan)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;33 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>13&#45;&gt;33</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M802.8913,-608.9735C816.9216,-596.2586 835.899,-579.0603 851.3936,-565.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"854.0747,-567.3121 859.1342,-558.0034 849.374,-562.1251 854.0747,-567.3121\"/>\n",
       "<text text-anchor=\"middle\" x=\"856\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advcl</text>\n",
       "</g>\n",
       "<!-- 37 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>37</title>\n",
       "<text text-anchor=\"middle\" x=\"1000\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">37 (.)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;37 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>13&#45;&gt;37</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M833.9532,-608.9355C870.2849,-595.7 920.0957,-576.8292 963,-558 963.0981,-557.9569 963.1964,-557.9138 963.2947,-557.8705\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"964.9878,-560.9451 972.6328,-553.6099 962.0821,-554.5767 964.9878,-560.9451\"/>\n",
       "<text text-anchor=\"middle\" x=\"940.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 39 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>39</title>\n",
       "<text text-anchor=\"middle\" x=\"1356\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">39 (berkata)</text>\n",
       "</g>\n",
       "<!-- 13&#45;&gt;39 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>13&#45;&gt;39</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M844.593,-617.6482C954.6095,-600.9441 1183.6759,-566.1644 1293.9914,-549.4149\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1294.6074,-552.8616 1303.9687,-547.9 1293.5565,-545.9409 1294.6074,-552.8616\"/>\n",
       "<text text-anchor=\"middle\" x=\"1117.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>2</title>\n",
       "<text text-anchor=\"middle\" x=\"81\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2 (Lumpur)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M209.4335,-521.9735C184.5627,-508.617 150.4799,-490.3133 123.6744,-475.9177\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"124.9894,-472.6512 114.5235,-471.0034 121.6775,-478.8182 124.9894,-472.6512\"/>\n",
       "<text text-anchor=\"middle\" x=\"184\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>4</title>\n",
       "<text text-anchor=\"middle\" x=\"243\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">4 (Ketua)</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;4 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M243,-521.9735C243,-510.1918 243,-494.5607 243,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"246.5001,-481.0033 243,-471.0034 239.5001,-481.0034 246.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"264.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>3</title>\n",
       "<text text-anchor=\"middle\" x=\"27\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3 (:)</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M69.8112,-434.9735C62.2087,-422.7252 52.024,-406.3165 43.4958,-392.5766\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"46.4219,-390.654 38.1745,-384.0034 40.4744,-394.3456 46.4219,-390.654\"/>\n",
       "<text text-anchor=\"middle\" x=\"78.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 5 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>5</title>\n",
       "<text text-anchor=\"middle\" x=\"135\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">5 (Penerangan)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;5 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>4&#45;&gt;5</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M220.6223,-434.9735C204.6934,-422.1419 183.0964,-404.7443 165.5784,-390.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"167.3322,-387.551 157.349,-384.0034 162.9409,-393.0023 167.3322,-387.551\"/>\n",
       "<text text-anchor=\"middle\" x=\"208\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 7 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>7</title>\n",
       "<text text-anchor=\"middle\" x=\"243\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">7 (,)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;7 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>4&#45;&gt;7</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M243,-434.9735C243,-423.1918 243,-407.5607 243,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"246.5001,-394.0033 243,-384.0034 239.5001,-394.0034 246.5001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"263.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 8 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>8</title>\n",
       "<text text-anchor=\"middle\" x=\"330\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">8 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;8 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>4&#45;&gt;8</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M267.6033,-434.8233C274.4538,-429.3668 281.7304,-423.1813 288,-417 295.7187,-409.39 303.462,-400.4488 310.1598,-392.1962\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"312.9933,-394.2562 316.4697,-384.2479 307.5108,-389.9039 312.9933,-394.2562\"/>\n",
       "<text text-anchor=\"middle\" x=\"323.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 6 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>6</title>\n",
       "<text text-anchor=\"middle\" x=\"135\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">6 (Bersatu)</text>\n",
       "</g>\n",
       "<!-- 5&#45;&gt;6 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>5&#45;&gt;6</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M135,-347.9735C135,-336.1918 135,-320.5607 135,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"138.5001,-307.0033 135,-297.0034 131.5001,-307.0034 138.5001,-307.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"147\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 9 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>9</title>\n",
       "<text text-anchor=\"middle\" x=\"326\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">9 (Wan)</text>\n",
       "</g>\n",
       "<!-- 8&#45;&gt;9 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>8&#45;&gt;9</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M329.1712,-347.9735C328.6295,-336.1918 327.9108,-320.5607 327.2946,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"330.7834,-306.832 326.8277,-297.0034 323.7908,-307.1536 330.7834,-306.832\"/>\n",
       "<text text-anchor=\"middle\" x=\"341\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 10 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>10</title>\n",
       "<text text-anchor=\"middle\" x=\"321\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">10 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 9&#45;&gt;10 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>9&#45;&gt;10</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M324.964,-260.9735C324.2869,-249.1918 323.3885,-233.5607 322.6183,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"326.1028,-219.786 322.0347,-210.0034 319.1143,-220.1877 326.1028,-219.786\"/>\n",
       "<text text-anchor=\"middle\" x=\"336\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 11 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>11</title>\n",
       "<text text-anchor=\"middle\" x=\"321\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">11 (Wan)</text>\n",
       "</g>\n",
       "<!-- 10&#45;&gt;11 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>10&#45;&gt;11</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M321,-173.9735C321,-162.1918 321,-146.5607 321,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"324.5001,-133.0033 321,-123.0034 317.5001,-133.0034 324.5001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"333\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 12 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>12</title>\n",
       "<text text-anchor=\"middle\" x=\"321\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">12 (Jan)</text>\n",
       "</g>\n",
       "<!-- 11&#45;&gt;12 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>11&#45;&gt;12</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M321,-86.9735C321,-75.1918 321,-59.5607 321,-46.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"324.5001,-46.0033 321,-36.0034 317.5001,-46.0034 324.5001,-46.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"333\" y=\"-57.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 15 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>15</title>\n",
       "<text text-anchor=\"middle\" x=\"390\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">15 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 14&#45;&gt;15 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>14&#45;&gt;15</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M467.8765,-521.9652C460.7805,-516.3073 453,-509.9769 446,-504 436.234,-495.6614 425.7698,-486.2631 416.5612,-477.8271\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"418.8901,-475.2137 409.1642,-471.0116 414.1468,-480.3617 418.8901,-475.2137\"/>\n",
       "<text text-anchor=\"middle\" x=\"458\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 19 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>19</title>\n",
       "<text text-anchor=\"middle\" x=\"492\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">19 (dan)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;19 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>20&#45;&gt;19</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M657.6951,-521.9735C623.378,-507.6884 575.4665,-487.7444 539.9861,-472.9751\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"540.9072,-469.5674 530.33,-468.9556 538.217,-476.0299 540.9072,-469.5674\"/>\n",
       "<text text-anchor=\"middle\" x=\"618\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 21 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>21</title>\n",
       "<text text-anchor=\"middle\" x=\"602\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">21 (Pemuda)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;21 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>20&#45;&gt;21</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M680.4871,-521.9735C665.8856,-509.1419 646.0884,-491.7443 630.0302,-477.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"632.3087,-474.9755 622.4866,-471.0034 627.6878,-480.2336 632.3087,-474.9755\"/>\n",
       "<text text-anchor=\"middle\" x=\"670\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 23 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>23</title>\n",
       "<text text-anchor=\"middle\" x=\"701\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">23 (,)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;23 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>20&#45;&gt;23</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M701,-521.9735C701,-510.1918 701,-494.5607 701,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"704.5001,-481.0033 701,-471.0034 697.5001,-481.0034 704.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"721.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 24 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>24</title>\n",
       "<text text-anchor=\"middle\" x=\"792\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">24 (Datuk)</text>\n",
       "</g>\n",
       "<!-- 20&#45;&gt;24 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>20&#45;&gt;24</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M725.4258,-521.6392C732.2735,-516.1798 739.593,-510.0387 746,-504 754.2852,-496.191 762.7893,-487.1123 770.1964,-478.7973\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"772.9607,-480.9534 776.919,-471.1257 767.696,-476.34 772.9607,-480.9534\"/>\n",
       "<text text-anchor=\"middle\" x=\"776\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 32 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>32</title>\n",
       "<text text-anchor=\"middle\" x=\"912\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">32 (kerajaan)</text>\n",
       "</g>\n",
       "<!-- 33&#45;&gt;32 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>33&#45;&gt;32</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M885.8376,-521.9735C890.3951,-509.9585 896.471,-493.9401 901.6203,-480.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"904.897,-481.5947 905.1711,-471.0034 898.352,-479.112 904.897,-481.5947\"/>\n",
       "<text text-anchor=\"middle\" x=\"918\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 34 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>34</title>\n",
       "<text text-anchor=\"middle\" x=\"1038\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">34 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 33&#45;&gt;34 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>33&#45;&gt;34</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M911.9449,-521.9735C936.3552,-508.617 969.8068,-490.3133 996.1159,-475.9177\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"998.0047,-478.8739 1005.0973,-471.0034 994.6446,-472.7331 998.0047,-478.8739\"/>\n",
       "<text text-anchor=\"middle\" x=\"980\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 38 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>38</title>\n",
       "<text text-anchor=\"middle\" x=\"1246\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">38 (Beliau)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;38 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>39&#45;&gt;38</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1333.2079,-521.9735C1316.8365,-509.0253 1294.5867,-491.4276 1276.6591,-477.2485\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1278.7774,-474.4616 1268.7629,-471.0034 1274.435,-479.952 1278.7774,-474.4616\"/>\n",
       "<text text-anchor=\"middle\" x=\"1328\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 41 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>41</title>\n",
       "<text text-anchor=\"middle\" x=\"1356\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">41 (Najib)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;41 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>39&#45;&gt;41</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1356,-521.9735C1356,-510.1918 1356,-494.5607 1356,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1359.5001,-481.0033 1356,-471.0034 1352.5001,-481.0034 1359.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1376\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 40 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>40</title>\n",
       "<text text-anchor=\"middle\" x=\"1445\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">40 (,)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;40 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>39&#45;&gt;40</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1379.4397,-521.9941C1386.2656,-516.4422 1393.6029,-510.1631 1400,-504 1408.1197,-496.1773 1416.4426,-487.0951 1423.6876,-478.7812\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1426.411,-480.982 1430.2617,-471.1117 1421.0963,-476.4263 1426.411,-480.982\"/>\n",
       "<text text-anchor=\"middle\" x=\"1435.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 64 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>64</title>\n",
       "<text text-anchor=\"middle\" x=\"1547\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">64 (berkata)</text>\n",
       "</g>\n",
       "<!-- 39&#45;&gt;64 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>39&#45;&gt;64</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1408.1022,-524.2056C1424.942,-518.511 1443.5142,-511.5953 1460,-504 1477.1476,-496.0998 1495.286,-485.8059 1510.5379,-476.5368\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1512.48,-479.4511 1519.1551,-471.2235 1508.8061,-473.4927 1512.48,-479.4511\"/>\n",
       "<text text-anchor=\"middle\" x=\"1501.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 16 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>16</title>\n",
       "<text text-anchor=\"middle\" x=\"430\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">16 (Seri)</text>\n",
       "</g>\n",
       "<!-- 15&#45;&gt;16 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>15&#45;&gt;16</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M398.288,-434.9735C403.8122,-422.9585 411.177,-406.9401 417.4186,-393.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"420.7252,-394.5512 421.7226,-384.0034 414.3652,-391.627 420.7252,-394.5512\"/>\n",
       "<text text-anchor=\"middle\" x=\"425\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 17 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>17</title>\n",
       "<text text-anchor=\"middle\" x=\"431\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">17 (Najib)</text>\n",
       "</g>\n",
       "<!-- 16&#45;&gt;17 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>16&#45;&gt;17</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M430.2072,-347.9735C430.3426,-336.1918 430.5223,-320.5607 430.6763,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"434.1778,-307.043 430.7931,-297.0034 427.1783,-306.9624 434.1778,-307.043\"/>\n",
       "<text text-anchor=\"middle\" x=\"443\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 18 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>18</title>\n",
       "<text text-anchor=\"middle\" x=\"431\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">18 (Razak)</text>\n",
       "</g>\n",
       "<!-- 17&#45;&gt;18 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>17&#45;&gt;18</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M431,-260.9735C431,-249.1918 431,-233.5607 431,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"434.5001,-220.0033 431,-210.0034 427.5001,-220.0034 434.5001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"443\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 22 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>22</title>\n",
       "<text text-anchor=\"middle\" x=\"602\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">22 (Umno)</text>\n",
       "</g>\n",
       "<!-- 21&#45;&gt;22 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>21&#45;&gt;22</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M602,-434.9735C602,-423.1918 602,-407.5607 602,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"605.5001,-394.0033 602,-384.0034 598.5001,-394.0034 605.5001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"614\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 25 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>25</title>\n",
       "<text text-anchor=\"middle\" x=\"790\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">25 (Dr)</text>\n",
       "</g>\n",
       "<!-- 24&#45;&gt;25 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>24&#45;&gt;25</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M791.5856,-434.9735C791.3148,-423.1918 790.9554,-407.5607 790.6473,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"794.1429,-393.9202 790.4139,-384.0034 787.1447,-394.0812 794.1429,-393.9202\"/>\n",
       "<text text-anchor=\"middle\" x=\"804\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 26 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>26</title>\n",
       "<text text-anchor=\"middle\" x=\"671\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">26 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 25&#45;&gt;26 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>25&#45;&gt;26</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M765.3431,-347.9735C747.5525,-334.9669 723.345,-317.269 703.9058,-303.0571\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"705.7636,-300.0798 695.6253,-297.0034 701.6323,-305.7307 705.7636,-300.0798\"/>\n",
       "<text text-anchor=\"middle\" x=\"750\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 30 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>30</title>\n",
       "<text text-anchor=\"middle\" x=\"820\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">30 (mempertikaikan)</text>\n",
       "</g>\n",
       "<!-- 25&#45;&gt;30 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>25&#45;&gt;30</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M796.216,-347.9735C800.3189,-336.0751 805.7756,-320.2508 810.4274,-306.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"813.8408,-307.5981 813.7919,-297.0034 807.2232,-305.3161 813.8408,-307.5981\"/>\n",
       "<text text-anchor=\"middle\" x=\"817.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 27 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>27</title>\n",
       "<text text-anchor=\"middle\" x=\"671\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">27 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 26&#45;&gt;27 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>26&#45;&gt;27</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M671,-260.9735C671,-249.1918 671,-233.5607 671,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"674.5001,-220.0033 671,-210.0034 667.5001,-220.0034 674.5001,-220.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 29 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>29</title>\n",
       "<text text-anchor=\"middle\" x=\"786\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">29 (yang)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;29 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>30&#45;&gt;29</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M812.9552,-260.9735C808.2596,-248.9585 801.9996,-232.9401 796.6942,-219.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"799.9357,-218.0434 793.0358,-210.0034 793.4159,-220.5914 799.9357,-218.0434\"/>\n",
       "<text text-anchor=\"middle\" x=\"826\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 31 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>31</title>\n",
       "<text text-anchor=\"middle\" x=\"902\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">31 (tindakan)</text>\n",
       "</g>\n",
       "<!-- 30&#45;&gt;31 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>30&#45;&gt;31</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M836.9905,-260.9735C848.8647,-248.3752 864.8875,-231.3755 878.0546,-217.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"880.7194,-219.6811 885.0313,-210.0034 875.6254,-214.8799 880.7194,-219.6811\"/>\n",
       "<text text-anchor=\"middle\" x=\"877\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 28 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>28</title>\n",
       "<text text-anchor=\"middle\" x=\"671\" y=\"-101.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">28 (Dusuki)</text>\n",
       "</g>\n",
       "<!-- 27&#45;&gt;28 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>27&#45;&gt;28</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M671,-173.9735C671,-162.1918 671,-146.5607 671,-133.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"674.5001,-133.0033 671,-123.0034 667.5001,-133.0034 674.5001,-133.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-144.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 35 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>35</title>\n",
       "<text text-anchor=\"middle\" x=\"908\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">35 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 34&#45;&gt;35 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>34&#45;&gt;35</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M992.6583,-434.8259C981.9008,-429.7047 970.7318,-423.683 961,-417 950.309,-409.6583 939.759,-400.1149 930.9208,-391.2722\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"933.41,-388.8115 923.9338,-384.0729 928.3867,-393.6866 933.41,-388.8115\"/>\n",
       "<text text-anchor=\"middle\" x=\"998\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 36 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>36</title>\n",
       "<text text-anchor=\"middle\" x=\"1039\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">36 (penuh)</text>\n",
       "</g>\n",
       "<!-- 34&#45;&gt;36 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>34&#45;&gt;36</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1038.2072,-434.9735C1038.3426,-423.1918 1038.5223,-407.5607 1038.6763,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1042.1778,-394.043 1038.7931,-384.0034 1035.1783,-393.9624 1042.1778,-394.043\"/>\n",
       "<text text-anchor=\"middle\" x=\"1059\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 46 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>46</title>\n",
       "<text text-anchor=\"middle\" x=\"1160\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">46 (memetik)</text>\n",
       "</g>\n",
       "<!-- 38&#45;&gt;46 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>38&#45;&gt;46</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1228.1807,-434.9735C1215.7272,-422.3752 1198.9229,-405.3755 1185.1135,-391.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1187.3157,-388.6547 1177.7964,-384.0034 1182.3374,-393.5757 1187.3157,-388.6547\"/>\n",
       "<text text-anchor=\"middle\" x=\"1222.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 61 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>61</title>\n",
       "<text text-anchor=\"middle\" x=\"1261\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">61 (.)</text>\n",
       "</g>\n",
       "<!-- 38&#45;&gt;61 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>38&#45;&gt;61</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1249.108,-434.9735C1251.1393,-423.1918 1253.8344,-407.5607 1256.1452,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1259.6459,-394.4527 1257.896,-384.0034 1252.7477,-393.2633 1259.6459,-394.4527\"/>\n",
       "<text text-anchor=\"middle\" x=\"1275.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 45 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>45</title>\n",
       "<text text-anchor=\"middle\" x=\"1017\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">45 (sengaja)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;45 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>46&#45;&gt;45</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1124.8624,-347.9535C1114.4841,-342.4005 1103.1807,-336.1309 1093,-330 1079.0645,-321.6079 1064.1206,-311.7718 1051.2034,-302.9888\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1052.8492,-299.8736 1042.6219,-297.1071 1048.8917,-305.6476 1052.8492,-299.8736\"/>\n",
       "<text text-anchor=\"middle\" x=\"1121.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 47 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>47</title>\n",
       "<text text-anchor=\"middle\" x=\"1149\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">47 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;47 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>46&#45;&gt;47</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1157.7208,-347.9735C1156.2311,-336.1918 1154.2548,-320.5607 1152.5602,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1156.0031,-306.4853 1151.2763,-297.0034 1149.0584,-307.3634 1156.0031,-306.4853\"/>\n",
       "<text text-anchor=\"middle\" x=\"1167\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 50 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>50</title>\n",
       "<text text-anchor=\"middle\" x=\"1256\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">50 (,)</text>\n",
       "</g>\n",
       "<!-- 46&#45;&gt;50 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>46&#45;&gt;50</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1179.8913,-347.9735C1193.9216,-335.2586 1212.899,-318.0603 1228.3936,-304.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1231.0747,-306.3121 1236.1342,-297.0034 1226.374,-301.1251 1231.0747,-306.3121\"/>\n",
       "<text text-anchor=\"middle\" x=\"1234.5\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 43 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>43</title>\n",
       "<text text-anchor=\"middle\" x=\"1356\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">43 (Asyraf)</text>\n",
       "</g>\n",
       "<!-- 41&#45;&gt;43 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>41&#45;&gt;43</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1356,-434.9735C1356,-423.1918 1356,-407.5607 1356,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1359.5001,-394.0033 1356,-384.0034 1352.5001,-394.0034 1359.5001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1371\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">conj</text>\n",
       "</g>\n",
       "<!-- 62 -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>62</title>\n",
       "<text text-anchor=\"middle\" x=\"1547\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">62 (Wan)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;62 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>64&#45;&gt;62</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1547,-434.9735C1547,-423.1918 1547,-407.5607 1547,-394.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1550.5001,-394.0033 1547,-384.0034 1543.5001,-394.0034 1550.5001,-394.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1567\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 83 -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>83</title>\n",
       "<text text-anchor=\"middle\" x=\"1632\" y=\"-362.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">83 (.)</text>\n",
       "</g>\n",
       "<!-- 64&#45;&gt;83 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>64&#45;&gt;83</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1571.0761,-434.8061C1577.7747,-429.3493 1584.8849,-423.1679 1591,-417 1598.5586,-409.3761 1606.1237,-400.4313 1612.6605,-392.1796\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1615.4589,-394.2824 1618.8162,-384.2336 1609.9251,-389.9955 1615.4589,-394.2824\"/>\n",
       "<text text-anchor=\"middle\" x=\"1625.5\" y=\"-405.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 42 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>42</title>\n",
       "<text text-anchor=\"middle\" x=\"1339\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">42 (dan)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;42 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>43&#45;&gt;42</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1352.4776,-347.9735C1350.1754,-336.1918 1347.121,-320.5607 1344.5022,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1347.8708,-306.1465 1342.5179,-297.0034 1341.0007,-307.489 1347.8708,-306.1465\"/>\n",
       "<text text-anchor=\"middle\" x=\"1357\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">cc</text>\n",
       "</g>\n",
       "<!-- 44 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>44</title>\n",
       "<text text-anchor=\"middle\" x=\"1440\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">44 (Wajdi)</text>\n",
       "</g>\n",
       "<!-- 43&#45;&gt;44 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>43&#45;&gt;44</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1373.4049,-347.9735C1385.5688,-335.3752 1401.9823,-318.3755 1415.4705,-304.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1418.1894,-306.6285 1422.6174,-297.0034 1413.1536,-301.7663 1418.1894,-306.6285\"/>\n",
       "<text text-anchor=\"middle\" x=\"1416\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 48 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>48</title>\n",
       "<text text-anchor=\"middle\" x=\"1051\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">48 (Perdana)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;48 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>47&#45;&gt;48</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1128.6943,-260.9735C1114.3717,-248.2586 1094.9989,-231.0603 1079.1815,-217.0183\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1081.0816,-214.0249 1071.2797,-210.0034 1076.4343,-219.2597 1081.0816,-214.0249\"/>\n",
       "<text text-anchor=\"middle\" x=\"1118\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 51 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>51</title>\n",
       "<text text-anchor=\"middle\" x=\"1161\" y=\"-188.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">51 (Tan)</text>\n",
       "</g>\n",
       "<!-- 47&#45;&gt;51 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>47&#45;&gt;51</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1151.4864,-260.9735C1153.1115,-249.1918 1155.2675,-233.5607 1157.1161,-220.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1160.6175,-220.3878 1158.5168,-210.0034 1153.6831,-219.4313 1160.6175,-220.3878\"/>\n",
       "<text text-anchor=\"middle\" x=\"1177.5\" y=\"-231.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">appos</text>\n",
       "</g>\n",
       "<!-- 49 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>49</title>\n",
       "<text text-anchor=\"middle\" x=\"1067\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">49 (Menteri)</text>\n",
       "</g>\n",
       "<!-- 49&#45;&gt;49 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>49&#45;&gt;49</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1120.6322,-721.5246C1131.0804,-720.6508 1138.5,-718.1426 1138.5,-714 1138.5,-711.3462 1135.455,-709.3631 1130.5195,-708.0507\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1131.0584,-704.5925 1120.6322,-706.4754 1129.9569,-711.5053 1131.0584,-704.5925\"/>\n",
       "<text text-anchor=\"middle\" x=\"1150.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 52 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>52</title>\n",
       "<text text-anchor=\"middle\" x=\"1468\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">52 (Sri)</text>\n",
       "</g>\n",
       "<!-- 52&#45;&gt;52 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>52&#45;&gt;52</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1503.7532,-721.8361C1513.7932,-721.4929 1521.5,-718.8809 1521.5,-714 1521.5,-710.9495 1518.4895,-708.7852 1513.7805,-707.5072\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1514.1294,-704.0227 1503.7532,-706.1639 1513.1999,-710.9608 1514.1294,-704.0227\"/>\n",
       "<text text-anchor=\"middle\" x=\"1533.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 57 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>57</title>\n",
       "<text text-anchor=\"middle\" x=\"1468\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">57 (lengkap)</text>\n",
       "</g>\n",
       "<!-- 52&#45;&gt;57 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>52&#45;&gt;57</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1468,-695.9735C1468,-684.1918 1468,-668.5607 1468,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1471.5001,-655.0033 1468,-645.0034 1464.5001,-655.0034 1471.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1488\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 55 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>55</title>\n",
       "<text text-anchor=\"middle\" x=\"1468\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">55 (yang)</text>\n",
       "</g>\n",
       "<!-- 57&#45;&gt;55 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>57&#45;&gt;55</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1468,-608.9735C1468,-597.1918 1468,-581.5607 1468,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1471.5001,-568.0033 1468,-558.0034 1464.5001,-568.0034 1471.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"1488\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 53 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>53</title>\n",
       "<text text-anchor=\"middle\" x=\"1627\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">53 (Muhyiddin)</text>\n",
       "</g>\n",
       "<!-- 53&#45;&gt;53 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>53&#45;&gt;53</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1690.1681,-721.3194C1700.7194,-720.2953 1708,-717.8555 1708,-714 1708,-711.5301 1705.012,-709.6412 1700.0904,-708.3333\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1700.6073,-704.8712 1690.1681,-706.6806 1699.4572,-711.7761 1700.6073,-704.8712\"/>\n",
       "<text text-anchor=\"middle\" x=\"1720\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 54 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>54</title>\n",
       "<text text-anchor=\"middle\" x=\"1798\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">54 (Yassin)</text>\n",
       "</g>\n",
       "<!-- 54&#45;&gt;54 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>54&#45;&gt;54</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1845.6902,-721.6424C1856.0134,-720.8951 1863.5,-718.3477 1863.5,-714 1863.5,-711.2827 1860.5756,-709.2686 1855.8619,-707.9578\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1856.1126,-704.4542 1845.6902,-706.3576 1855.0248,-711.3692 1856.1126,-704.4542\"/>\n",
       "<text text-anchor=\"middle\" x=\"1875.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 56 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>56</title>\n",
       "<text text-anchor=\"middle\" x=\"1897\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">56 (tidak)</text>\n",
       "</g>\n",
       "<!-- 58 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>58</title>\n",
       "<text text-anchor=\"middle\" x=\"1951\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">58 (untuk)</text>\n",
       "</g>\n",
       "<!-- 58&#45;&gt;56 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>58&#45;&gt;56</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1937.9586,-695.8998C1934.0084,-690.2392 1929.7305,-683.9237 1926,-678 1921.1625,-670.3185 1916.1784,-661.8041 1911.7361,-653.9677\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1914.7501,-652.1866 1906.8113,-645.1706 1908.6421,-655.6061 1914.7501,-652.1866\"/>\n",
       "<text text-anchor=\"middle\" x=\"1954.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 58&#45;&gt;58 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>58&#45;&gt;58</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1996.6646,-721.6827C2006.9656,-720.9906 2014.5,-718.4297 2014.5,-714 2014.5,-711.2314 2011.5569,-709.1929 2006.8346,-707.8843\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2007.081,-704.381 1996.6646,-706.3173 2006.0149,-711.2994 2007.081,-704.381\"/>\n",
       "<text text-anchor=\"middle\" x=\"2030.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 59 -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>59</title>\n",
       "<text text-anchor=\"middle\" x=\"2032\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">59 (mengelirukan)</text>\n",
       "</g>\n",
       "<!-- 58&#45;&gt;59 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>58&#45;&gt;59</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1967.7833,-695.9735C1979.5127,-683.3752 1995.3401,-666.3755 2008.3466,-652.4055\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2010.9857,-654.7073 2015.2382,-645.0034 2005.8624,-649.9374 2010.9857,-654.7073\"/>\n",
       "<text text-anchor=\"middle\" x=\"2020.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">xcomp</text>\n",
       "</g>\n",
       "<!-- 60 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>60</title>\n",
       "<text text-anchor=\"middle\" x=\"2032\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">60 (rakyat)</text>\n",
       "</g>\n",
       "<!-- 59&#45;&gt;60 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>59&#45;&gt;60</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2032,-608.9735C2032,-597.1918 2032,-581.5607 2032,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2035.5001,-568.0033 2032,-558.0034 2028.5001,-568.0034 2035.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2043\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 63 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>63</title>\n",
       "<text text-anchor=\"middle\" x=\"1548\" y=\"-275.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">63 (Saiful)</text>\n",
       "</g>\n",
       "<!-- 62&#45;&gt;63 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>62&#45;&gt;63</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1547.2072,-347.9735C1547.3426,-336.1918 1547.5223,-320.5607 1547.6763,-307.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1551.1778,-307.043 1547.7931,-297.0034 1544.1783,-306.9624 1551.1778,-307.043\"/>\n",
       "<text text-anchor=\"middle\" x=\"1560\" y=\"-318.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">flat</text>\n",
       "</g>\n",
       "<!-- 65 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>65</title>\n",
       "<text text-anchor=\"middle\" x=\"2260\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">65 (,)</text>\n",
       "</g>\n",
       "<!-- 65&#45;&gt;65 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>65&#45;&gt;65</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2287.2408,-721.8673C2297.0239,-722.0365 2305,-719.4141 2305,-714 2305,-710.7008 2302.0382,-708.4382 2297.5105,-707.2123\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2297.5519,-703.6975 2287.2408,-706.1327 2296.82,-710.6591 2297.5519,-703.6975\"/>\n",
       "<text text-anchor=\"middle\" x=\"2325.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">punct</text>\n",
       "</g>\n",
       "<!-- 68 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>68</title>\n",
       "<text text-anchor=\"middle\" x=\"2212\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">68 (menjangka)</text>\n",
       "</g>\n",
       "<!-- 65&#45;&gt;68 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>65&#45;&gt;68</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2250.0544,-695.9735C2243.361,-683.8418 2234.4158,-667.6287 2226.8799,-653.9698\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2229.8282,-652.0684 2221.9329,-645.0034 2223.6992,-655.4499 2229.8282,-652.0684\"/>\n",
       "<text text-anchor=\"middle\" x=\"2252.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">dep</text>\n",
       "</g>\n",
       "<!-- 69 -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>69</title>\n",
       "<text text-anchor=\"middle\" x=\"2332\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">69 (ada)</text>\n",
       "</g>\n",
       "<!-- 65&#45;&gt;69 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>65&#45;&gt;69</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2274.9184,-695.9735C2285.2481,-683.4919 2299.1532,-666.6899 2310.6529,-652.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2313.4213,-654.9388 2317.1007,-645.0034 2308.0286,-650.4758 2313.4213,-654.9388\"/>\n",
       "<text text-anchor=\"middle\" x=\"2324.5\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">xcomp</text>\n",
       "</g>\n",
       "<!-- 66 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>66</title>\n",
       "<text text-anchor=\"middle\" x=\"2178\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">66 (beliau)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;66 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>68&#45;&gt;66</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2204.9552,-608.9735C2200.2596,-596.9585 2193.9996,-580.9401 2188.6942,-567.3646\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2191.9357,-566.0434 2185.0358,-558.0034 2185.4159,-568.5914 2191.9357,-566.0434\"/>\n",
       "<text text-anchor=\"middle\" x=\"2218\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">nsubj</text>\n",
       "</g>\n",
       "<!-- 67 -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>67</title>\n",
       "<text text-anchor=\"middle\" x=\"2289\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">67 (sudah)</text>\n",
       "</g>\n",
       "<!-- 68&#45;&gt;67 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>68&#45;&gt;67</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2227.9544,-608.9735C2239.0014,-596.4919 2253.8722,-579.6899 2266.1705,-565.7944\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2269.0593,-567.8114 2273.066,-558.0034 2263.8174,-563.172 2269.0593,-567.8114\"/>\n",
       "<text text-anchor=\"middle\" x=\"2284.5\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">advmod</text>\n",
       "</g>\n",
       "<!-- 70 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>70</title>\n",
       "<text text-anchor=\"middle\" x=\"2443\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">70 (kenyataan)</text>\n",
       "</g>\n",
       "<!-- 70&#45;&gt;70 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>70&#45;&gt;70</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2504.6667,-721.3516C2515.2013,-720.347 2522.5,-717.8965 2522.5,-714 2522.5,-711.5038 2519.5046,-709.6011 2514.582,-708.2917\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2515.1045,-704.8306 2504.6667,-706.6484 2513.9599,-711.7364 2515.1045,-704.8306\"/>\n",
       "<text text-anchor=\"middle\" x=\"2533.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 71 -->\n",
       "<g id=\"node73\" class=\"node\">\n",
       "<title>71</title>\n",
       "<text text-anchor=\"middle\" x=\"2443\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">71 (balas)</text>\n",
       "</g>\n",
       "<!-- 70&#45;&gt;71 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>70&#45;&gt;71</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2443,-695.9735C2443,-684.1918 2443,-668.5607 2443,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2446.5001,-655.0033 2443,-645.0034 2439.5001,-655.0034 2446.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2466\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">ccomp</text>\n",
       "</g>\n",
       "<!-- 73 -->\n",
       "<g id=\"node74\" class=\"node\">\n",
       "<title>73</title>\n",
       "<text text-anchor=\"middle\" x=\"2420\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">73 (Najib)</text>\n",
       "</g>\n",
       "<!-- 71&#45;&gt;73 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>71&#45;&gt;73</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2438.2344,-608.9735C2435.0888,-597.0751 2430.9054,-581.2508 2427.339,-567.7606\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2430.6992,-566.7766 2424.7595,-558.0034 2423.9317,-568.5658 2430.6992,-566.7766\"/>\n",
       "<text text-anchor=\"middle\" x=\"2444\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 75 -->\n",
       "<g id=\"node75\" class=\"node\">\n",
       "<title>75</title>\n",
       "<text text-anchor=\"middle\" x=\"2547\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">75 (tulisan)</text>\n",
       "</g>\n",
       "<!-- 71&#45;&gt;75 -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>71&#45;&gt;75</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2464.5489,-608.9735C2479.8878,-596.1419 2500.6849,-578.7443 2517.5541,-564.6326\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2520.0543,-567.1043 2525.4787,-558.0034 2515.5629,-561.7352 2520.0543,-567.1043\"/>\n",
       "<text text-anchor=\"middle\" x=\"2513\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obl</text>\n",
       "</g>\n",
       "<!-- 72 -->\n",
       "<g id=\"node76\" class=\"node\">\n",
       "<title>72</title>\n",
       "<text text-anchor=\"middle\" x=\"2286\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">72 (daripada)</text>\n",
       "</g>\n",
       "<!-- 73&#45;&gt;72 -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>73&#45;&gt;72</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2392.2351,-521.9735C2371.9324,-508.792 2344.2084,-490.792 2322.173,-476.4855\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2324.0226,-473.5133 2313.7293,-471.0034 2320.2107,-479.3844 2324.0226,-473.5133\"/>\n",
       "<text text-anchor=\"middle\" x=\"2378\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 74 -->\n",
       "<g id=\"node77\" class=\"node\">\n",
       "<title>74</title>\n",
       "<text text-anchor=\"middle\" x=\"2420\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">74 (mengenai)</text>\n",
       "</g>\n",
       "<!-- 73&#45;&gt;74 -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>73&#45;&gt;74</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2420,-521.9735C2420,-510.1918 2420,-494.5607 2420,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2423.5001,-481.0033 2420,-471.0034 2416.5001,-481.0034 2423.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2436\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">case</text>\n",
       "</g>\n",
       "<!-- 77 -->\n",
       "<g id=\"node78\" class=\"node\">\n",
       "<title>77</title>\n",
       "<text text-anchor=\"middle\" x=\"2562\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">77 (berhubung)</text>\n",
       "</g>\n",
       "<!-- 75&#45;&gt;77 -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>75&#45;&gt;77</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2550.108,-521.9735C2552.1393,-510.1918 2554.8344,-494.5607 2557.1452,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2560.6459,-481.4527 2558.896,-471.0034 2553.7477,-480.2633 2560.6459,-481.4527\"/>\n",
       "<text text-anchor=\"middle\" x=\"2566.5\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">acl</text>\n",
       "</g>\n",
       "<!-- 76 -->\n",
       "<g id=\"node79\" class=\"node\">\n",
       "<title>76</title>\n",
       "<text text-anchor=\"middle\" x=\"2809\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">76 (beliau)</text>\n",
       "</g>\n",
       "<!-- 76&#45;&gt;76 -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>76&#45;&gt;76</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2855.6737,-721.663C2865.988,-720.943 2873.5,-718.3887 2873.5,-714 2873.5,-711.2571 2870.5656,-709.2307 2865.8467,-707.9208\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2866.0931,-704.4171 2855.6737,-706.337 2865.0161,-711.3338 2866.0931,-704.4171\"/>\n",
       "<text text-anchor=\"middle\" x=\"2885.5\" y=\"-710.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">det</text>\n",
       "</g>\n",
       "<!-- 78 -->\n",
       "<g id=\"node80\" class=\"node\">\n",
       "<title>78</title>\n",
       "<text text-anchor=\"middle\" x=\"2809\" y=\"-623.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">78 (kesan)</text>\n",
       "</g>\n",
       "<!-- 76&#45;&gt;78 -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>76&#45;&gt;78</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2809,-695.9735C2809,-684.1918 2809,-668.5607 2809,-655.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2812.5001,-655.0033 2809,-645.0034 2805.5001,-655.0034 2812.5001,-655.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2820\" y=\"-666.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">obj</text>\n",
       "</g>\n",
       "<!-- 79 -->\n",
       "<g id=\"node81\" class=\"node\">\n",
       "<title>79</title>\n",
       "<text text-anchor=\"middle\" x=\"2691\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">79 (positif)</text>\n",
       "</g>\n",
       "<!-- 78&#45;&gt;79 -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>78&#45;&gt;79</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2784.5503,-608.9735C2766.9092,-595.9669 2742.9051,-578.269 2723.6292,-564.0571\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2725.5442,-561.1206 2715.4184,-558.0034 2721.3902,-566.7548 2725.5442,-561.1206\"/>\n",
       "<text text-anchor=\"middle\" x=\"2778\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "<!-- 80 -->\n",
       "<g id=\"node82\" class=\"node\">\n",
       "<title>80</title>\n",
       "<text text-anchor=\"middle\" x=\"2809\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">80 (sekatan)</text>\n",
       "</g>\n",
       "<!-- 78&#45;&gt;80 -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>78&#45;&gt;80</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2809,-608.9735C2809,-597.1918 2809,-581.5607 2809,-568.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2812.5001,-568.0033 2809,-558.0034 2805.5001,-568.0034 2812.5001,-568.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2846\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 81 -->\n",
       "<g id=\"node83\" class=\"node\">\n",
       "<title>81</title>\n",
       "<text text-anchor=\"middle\" x=\"2945\" y=\"-536.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">81 (pergerakan)</text>\n",
       "</g>\n",
       "<!-- 78&#45;&gt;81 -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>78&#45;&gt;81</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2853.9114,-608.9585C2865.0623,-603.7604 2876.7491,-597.6698 2887,-591 2898.44,-583.5565 2909.9522,-573.9693 2919.6706,-565.12\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2922.1229,-567.6187 2927.0443,-558.2362 2917.3461,-562.5019 2922.1229,-567.6187\"/>\n",
       "<text text-anchor=\"middle\" x=\"2942\" y=\"-579.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">compound</text>\n",
       "</g>\n",
       "<!-- 82 -->\n",
       "<g id=\"node84\" class=\"node\">\n",
       "<title>82</title>\n",
       "<text text-anchor=\"middle\" x=\"2691\" y=\"-449.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">82 (penuh)</text>\n",
       "</g>\n",
       "<!-- 79&#45;&gt;82 -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>79&#45;&gt;82</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M2691,-521.9735C2691,-510.1918 2691,-494.5607 2691,-481.1581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2694.5001,-481.0033 2691,-471.0034 2687.5001,-481.0034 2694.5001,-481.0033\"/>\n",
       "<text text-anchor=\"middle\" x=\"2711\" y=\"-492.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">amod</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x7f32744c0cf8>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = 'KUALA LUMPUR: Ketua Penerangan BERSATU, Datuk Wan Saiful Wan Jan membidas kenyataan Datuk Seri Najib Razak dan Ketua Pemuda UMNO, Datuk Dr Asyraf Wajdi Dusuki yang mempertikaikan tindakan kerajaan melaksanakan sekatan pergerakan penuh. Beliau berkata, Najib dan Asyraf Wajdi sengaja memetik kenyataan Perdana Menteri, Tan Sri Muhyiddin Yassin yang tidak lengkap untuk mengelirukan rakyat. Wan Saiful berkata, beliau sudah menjangka ada kenyataan balas daripada Najib mengenai tulisan beliau berhubung kesan positif sekatan pergerakan penuh.'\n",
    "sequence = transformer_textcleaning(string)[1]\n",
    "parsed_sequence, bert_sequence, mask = parse_X(sequence)\n",
    "h, t = sess.run([model.heads_seq, model.tags_seq],\n",
    "        feed_dict = {\n",
    "            model.words: [parsed_sequence],\n",
    "        },\n",
    ")\n",
    "h = h[0] - 2\n",
    "t = [idx2tag[d] for d in t[0]]\n",
    "merged_h = merge_sentencepiece_tokens_tagging(bert_sequence, h)\n",
    "merged_t = merge_sentencepiece_tokens_tagging(bert_sequence, t)\n",
    "tagging = list(zip(merged_t[0], merged_t[1]))\n",
    "indexing = list(zip(merged_h[0], merged_h[1]))\n",
    "dep = dependency_graph(tagging, indexing)\n",
    "dep.to_graphvis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tiny-bert-dependency/model.ckpt'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'tiny-bert-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from tiny-bert-dependency/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "\n",
    "learning_rate = 2e-5\n",
    "hidden_size_word = 256\n",
    "\n",
    "model = Model(learning_rate, hidden_size_word, training = False)\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'tiny-bert-dependency/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p])\n",
    "        out.append(out_i)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(heads_pred, types_pred, heads, types, lengths,\n",
    "             symbolic_root=False, symbolic_end=False):\n",
    "    batch_size, _ = heads_pred.shape\n",
    "    ucorr = 0.\n",
    "    lcorr = 0.\n",
    "    total = 0.\n",
    "    ucomplete_match = 0.\n",
    "    lcomplete_match = 0.\n",
    "\n",
    "    corr_root = 0.\n",
    "    total_root = 0.\n",
    "    start = 1 if symbolic_root else 0\n",
    "    end = 1 if symbolic_end else 0\n",
    "    for i in range(batch_size):\n",
    "        ucm = 1.\n",
    "        lcm = 1.\n",
    "        for j in range(start, lengths[i] - end):\n",
    "\n",
    "            total += 1\n",
    "            if heads[i, j] == heads_pred[i, j]:\n",
    "                ucorr += 1\n",
    "                if types[i, j] == types_pred[i, j]:\n",
    "                    lcorr += 1\n",
    "                else:\n",
    "                    lcm = 0\n",
    "            else:\n",
    "                ucm = 0\n",
    "                lcm = 0\n",
    "\n",
    "            if heads[i, j] == 0:\n",
    "                total_root += 1\n",
    "                corr_root += 1 if heads_pred[i, j] == 0 else 0\n",
    "\n",
    "        ucomplete_match += ucm\n",
    "        lcomplete_match += lcm\n",
    "    \n",
    "    return ucorr / total, lcorr / total, corr_root / total_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 313/313 [00:38<00:00,  8.12it/s]\n"
     ]
    }
   ],
   "source": [
    "arcs, types, roots = [], [], []\n",
    "real_Y, predict_Y = [], []\n",
    "\n",
    "for i in tqdm(range(0, len(test_X), batch_size)):\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = test_X[i: index]\n",
    "    batch_x = pad_sequences(batch_x,padding='post')\n",
    "    batch_y = test_Y[i: index]\n",
    "    batch_y = pad_sequences(batch_y,padding='post')\n",
    "    batch_depends = test_depends[i: index]\n",
    "    batch_depends = pad_sequences(batch_depends,padding='post')\n",
    "    \n",
    "    tags_seq, heads = sess.run(\n",
    "        [model.logits, model.heads_seq],\n",
    "        feed_dict = {\n",
    "            model.words: batch_x,\n",
    "        },\n",
    "    )\n",
    "    \n",
    "    arc_accuracy, type_accuracy, root_accuracy = evaluate(heads - 1, tags_seq, batch_depends - 1, batch_y, \n",
    "            np.count_nonzero(batch_x, axis = 1))\n",
    "    arcs.append(arc_accuracy)\n",
    "    types.append(type_accuracy)\n",
    "    roots.append(root_accuracy)\n",
    "    predicted = pred2label(tags_seq)\n",
    "    real = pred2label(batch_y)\n",
    "    predict_Y.extend(predicted)\n",
    "    real_Y.extend(real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_real_Y = []\n",
    "for r in real_Y:\n",
    "    temp_real_Y.extend(r)\n",
    "    \n",
    "temp_predict_Y = []\n",
    "for r in predict_Y:\n",
    "    temp_predict_Y.extend(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "          PAD    0.99958   1.00000   0.99979    656062\n",
      "            X    0.99998   0.99232   0.99613     61945\n",
      "          acl    0.64541   0.81625   0.72085      3298\n",
      "        advcl    0.27776   0.56039   0.37142      1656\n",
      "       advmod    0.91310   0.92373   0.91839      6700\n",
      "         amod    0.82278   0.86302   0.84242      4621\n",
      "        appos    0.76714   0.69299   0.72818      3052\n",
      "         case    0.96439   0.97337   0.96886     11492\n",
      "           cc    0.98187   0.96747   0.97461      3750\n",
      "        ccomp    0.45118   0.34987   0.39412       383\n",
      "     compound    0.83453   0.89652   0.86442     11133\n",
      "compound:plur    0.43478   0.33333   0.37736        30\n",
      "         conj    0.84660   0.85878   0.85265      5424\n",
      "          cop    0.97119   0.95659   0.96384       599\n",
      "        csubj    0.25000   0.10000   0.14286        10\n",
      "   csubj:pass    0.00000   0.00000   0.00000         1\n",
      "          dep    0.45412   0.53168   0.48985       363\n",
      "          det    0.92230   0.89720   0.90958      3969\n",
      "        fixed    0.88889   0.70175   0.78431       171\n",
      "         flat    0.91953   0.94618   0.93266     18393\n",
      "         iobj    0.00000   0.00000   0.00000         4\n",
      "         mark    0.88054   0.91620   0.89802      1778\n",
      "         nmod    0.77113   0.77499   0.77306      4591\n",
      "        nsubj    0.59707   0.82001   0.69100      7145\n",
      "   nsubj:pass    0.74934   0.69813   0.72283      2034\n",
      "       nummod    0.89096   0.94680   0.91803      4436\n",
      "          obj    0.88715   0.87539   0.88123      6412\n",
      "          obl    0.83020   0.74976   0.78793      5191\n",
      "    parataxis    0.09302   0.19355   0.12565       372\n",
      "        punct    0.99385   0.99501   0.99443     20643\n",
      "         root    0.45797   0.14820   0.22393     10000\n",
      "        xcomp    0.56806   0.68743   0.62207      1718\n",
      "\n",
      "     accuracy                        0.97313    857376\n",
      "    macro avg    0.68951   0.69272   0.68345    857376\n",
      " weighted avg    0.97255   0.97313   0.97156    857376\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(temp_real_Y, temp_predict_Y, digits = 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arc accuracy: 0.795252499643322\n",
      "types accuracy: 0.7247015428088897\n",
      "root accuracy: 0.9893913291696776\n"
     ]
    }
   ],
   "source": [
    "print('arc accuracy:', np.mean(arcs))\n",
    "print('types accuracy:', np.mean(types))\n",
    "print('root accuracy:', np.mean(roots))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or '_seq' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'adam' not in n.name\n",
    "        and 'gradients/bert' not in n.name\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from tiny-bert-dependency/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-32-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 86 variables.\n",
      "INFO:tensorflow:Converted 86 variables to const ops.\n",
      "1400 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('tiny-bert-dependency', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-35-b945beb65a73>:6: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "tf.set_random_seed(0)\n",
    "\n",
    "pb = 'tiny-bert-dependency/frozen_model.pb'\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "\n",
    "if 'bert' in pb:\n",
    "    inputs = ['Placeholder']\n",
    "    a = ['dense/BiasAdd']\n",
    "if 'xlnet' in pb:\n",
    "    inputs = ['Placeholder', 'Placeholder_1', 'Placeholder_2']\n",
    "    a = ['transpose_3']\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       inputs,\n",
    "                                       ['logits', 'heads_seq'] + a, transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
